#!/usr/bin/env python  
#coding=utf-8

import urllib2, urllib, re, os, time
import sqlite3
from BeautifulSoup import BeautifulSoup

##函数
#解析页面函数
#getpage
def getPage(url):
  try:
    page = urllib2.urlopen(url) 
    soup = BeautifulSoup(page) 
    return soup
  except:
    return 0
  
#去掉HTML标签
def dropTags(soupStr):
  soupStr = str(soupStr)
  soupStr = re.compile('<[^>]+>').sub("",soupStr)
  soupStr = re.compile('&nbsp;').sub("",soupStr)
  soupStr = re.compile('&copy;').sub("",soupStr)
  return re.compile('豆瓣').sub("",soupStr)

  
##Begin！
#设置当前目录为工作目录
os.chdir(os.path.dirname(__file__))

#数据库连接
conn = sqlite3.connect('doubanMoviesDB.db')
curs = conn.cursor()

#豆瓣电影网址
webUrl = 'http://movie.douban.com/subject/'

#http://movie.douban.com/subject/3731581/
############4707585   音乐3262053
subjectId = 1002558

while subjectId <= 9999999:
  
  time.sleep(1)
  
  subjectId += 1
  
  print subjectId
  
  subjectUrl = 'http://movie.douban.com/subject/'+str(subjectId)
  
  if getPage(subjectUrl) == 0:
    
    continue
    
  else:
    
    subjectSoup = getPage(subjectUrl)
      
    if type(subjectSoup) == int:
      subjectId -= 1
      continue
    else:
      if subjectSoup.find('li', {'class':'on'}).find('a').string != '豆瓣电影':
        continue
      elif subjectSoup.find('span', {'property':'v:summary'}):
        
        h1Span = subjectSoup.find('span', {'property':'v:itemreviewed'})
        
        movieName = h1Span.string

        print movieName

        if subjectSoup.find('span', {'class':'all hidden'}):
          soupStr = str(subjectSoup.find('span', {'class':'all hidden'}))
          description = dropTags(soupStr)
          print description
          
        else:
          soupStr = str(subjectSoup.find('span', {'property':'v:summary'}))
          description = dropTags(soupStr)
          print description
        
        query = "insert into movies(name,subjectid,description) values ('"+ movieName +"','" + str(subjectId) +"','"+ description +"')"
        curs.execute(query)
        conn.commit()
      else:
        continue
    
conn.close()


